from io import BytesIO
from PIL import Image
import pymongo
import time
from aip import AipOcr
from aip import AipNlp


class movieSpider:

    def __init__(self):
        #连接百度Ocr
        APP_ID = '24217351'
        API_KEY = '7vRbBsMqcMPyFqry1nxwTdrH'
        SECRET_KEY = 'LqoitsxibGBdy149lHlIOzBueTEHE8Nv'
        self.client = AipOcr(APP_ID, API_KEY, SECRET_KEY)
        #连接百度Nlp
        APP_ID = '24235922'
        API_KEY = 'PZTrojiCEybGWoiK46by5Uo0'
        SECRET_KEY = '1tkwHVIjMSgl90t0IL0r1FdNSQUn0Gek'
        self.client_nlp = AipNlp(APP_ID, API_KEY, SECRET_KEY)
        #连接数据库
        myclient = pymongo.MongoClient('mongodb://47.93.220.108:27017/')
        self.mydb = myclient['movie']
    

    #图片转文字
    def get_word(self,path):
        print('正在向baiduAip请求文字识别结果...')
        img = open(path, 'rb').read()
        options = {}
        options["language_type"] = "CHN_ENG"
        res = self.client.basicGeneral(img, options)
        res = res.get('words_result')[0].get('words')
        return res

    #情感分析
    def get_sentiment(self,text):
        try:
            res = self.client_nlp.sentimentClassify(text)
            res = res.get('items')[0].get('sentiment')
        except BaseException:
            res = -1    
        return res


    #一个小方法，用于判断网页元素是否存在
    def isExist(self,ele_list):
        if(len(ele_list) == 0):
            return False
        return True

    #一个小方法，判断该电影是否已经存在于数据库？
    def isInDB(self,movie_name,table_name):
        count = table_name.count_documents({'movie_name':movie_name})
        #不存在，则开始获取
        if(count != 0):
            print('该电影已收录')
            return True
        return False
        pass
    
    ##核心方法，爬取某天的电影票房，上座率，直接存入数据库中
    def getMovieDataByDate(self,driver,date,driver2,signal=0):

        if(signal == 1):
            self.mydb.alter.delete_many({})
            print('alter数据已清除')
        
        print('**** 开始爬取'+date+'的电影数据，数据来源：猫眼电影 ***')

        driver.get('https://piaofang.maoyan.com/dashboard/movie?date=' + date)
        #等待页面加载完成
        time.sleep(3)
        #####
        web_element = driver.find_element_by_xpath('//div[@class="movielist"]/table/tbody')
        #获取本日上映电影名,票房,上座率
        movie_names = [ movie_name.get_attribute('title') for movie_name in web_element.find_elements_by_xpath('tr/td[1]')]
        #获取本日上映电影票房截图
        boxOffice_scs = []
        for boxOffice in web_element.find_elements_by_xpath('tr/td[2]'):
            #控制滚动条与底部对齐，否则截图会出现偏移
            driver.execute_script("arguments[0].scrollIntoView(false);",boxOffice)
            boxOffice_scs.append(boxOffice.screenshot_as_png)
        #这个list存本日上映电影票房
        boxOffices = []
        #票房占比
        boxOffice_percentages =[ boxOffice_percentage.text for boxOffice_percentage in web_element.find_elements_by_xpath('tr/td[3]')]
        #排片场次
        arranges = [ arrange.text for arrange in web_element.find_elements_by_xpath('tr/td[4]')]
        #排片占比
        arrange_percentages = [ arrange_percentage.text for arrange_percentage in web_element.find_elements_by_xpath('tr/td[5]')]

        #获取本日上映电影上座率
        attendances = [ attendance.text for attendance in web_element.find_elements_by_xpath('tr/td[@class="last-tab"]')]
        for boxOffice_sc in boxOffice_scs:
            Image.open(BytesIO(boxOffice_sc)).save('boxOffice.png')
            boxOffices.append(self.get_word('boxOffice.png'))
        
        #数据粗处理，以及打包整理
        
        for movie_name, boxOffice, boxOffice_percentage, arrange, arrange_percentage, attendance in zip(movie_names,boxOffices,boxOffice_percentages,arranges,arrange_percentages,attendances):
            #针对识别不出小数点的情况
            if boxOffice[-4] != '.':
                temp = list(boxOffice)
                temp.insert(-3,'.')
                boxOffice = ''.join(temp)

            data = {
                'date':date,
                'boxOffice' : boxOffice,
                'boxOffice_percentage':boxOffice_percentage,
                'arrange':arrange,
                'arrange_percentage':arrange_percentage,  
                'attendance' : attendance
            }
            #爬取该电影的简介信息
            self.getMovieMsgByMovieName(driver=driver2, movie_name=movie_name, own_data=data,signal=signal)
        print('～～'+ date +' 爬取完成 ～～')
        pass

    ##核心方法，爬取某个电影的简介信息，以及短评等内容
    def getMovieMsgByMovieName(self,driver,movie_name,own_data,signal):
        #该电影是否存在？
        if(self.isInDB(movie_name,self.mydb.movieBaseInfo)):
            dt = self.mydb.movieBaseInfo.find({'movie_name':movie_name})
            a = list(dt)[0]
            data = {
            'date':own_data["date"],
            'movie_name':movie_name,
            'movie_img':a['movie_img'],
            'boxOffice':own_data["boxOffice"],
            'boxOffice_percentage':own_data['boxOffice_percentage'],
            'arrange':own_data['arrange'],
            'arrange_percentage':own_data['arrange_percentage'],  
            'attendance':own_data["attendance"],
            }
            self.mydb.movieInfo.insert(data)
            if(signal == 1):
                self.mydb.alter.insert(data)
            return
        driver.get('https://search.douban.com/movie/subject_search?search_text='+movie_name)
        #等待页面加载完成
        time.sleep(1)
        #判断是否有结果
        if(not self.isExist(driver.find_elements_by_xpath('//div[@class="title"]/a'))):
            print('无结果')
            return

        #爬取流程
        if('celebrity' in driver.find_element_by_xpath('//div[@class="title"]/a').get_attribute('href')):
            driver.find_elements_by_xpath('//div[@class="title"]/a')[1].click()
        else:
            driver.find_element_by_xpath('//div[@class="title"]/a').click()
        time.sleep(1)

        #电影封面
        movie_img = driver.find_elements_by_xpath('//div[@id="mainpic"]/a/img')
        if(self.isExist(movie_img)):
            movie_img = movie_img[0].get_attribute('src')
        else:
            movie_img = ''

        #电影导演
        movie_director = driver.find_elements_by_xpath('//div[@id="info"]/span/span[2]/a')
        if(self.isExist(movie_director)):
            movie_director = movie_director[0].text
        else:
            movie_director = '未知'
        
        #电影明星
        movie_star_info = []
        movie_star_list = driver.find_elements_by_xpath("//li[@class='celebrity']")
        if(self.isExist(movie_star_list)):
            for item in movie_star_list:
                movie_star_data = {
                    "movie_star_name":item.find_element_by_css_selector("a").get_attribute('title'),
                    "movie_star_img":item.find_element_by_css_selector("a > div").get_attribute('style').split("\"")[1],
                    "movie_star_role":item.find_element_by_css_selector("span.role").get_attribute('title')
                }
                movie_star_info.append(movie_star_data)
        
        #电影类型
        movie_class_list = driver.find_elements_by_xpath('//div[@id="info"]/span[@property="v:genre"]')
        if(self.isExist(movie_class_list)):
            movie_class = ','.join(genre.text for genre in movie_class_list)
        else:
            movie_class = '未知'

        #上映日期
        movie_date = driver.find_elements_by_xpath('//div[@id="info"]/span[@property="v:initialReleaseDate"]')
        if(self.isExist(movie_date)):
            movie_date = movie_date[0].text
        else:
            movie_date = '未知'
        #国家
        movie_country = driver.find_elements_by_xpath("//div[@id='info']")
        if(self.isExist(movie_country)):
            movie_country = (movie_country[0].text.split("制片国家/地区: "))[1].split("\n")[0]
        else:
            movie_country = '未知'
        #电影时长
        movie_length = driver.find_elements_by_xpath('//div[@id="info"]/span[@property="v:runtime"]')
        if(self.isExist(movie_length)):
            movie_length = movie_length[0].text
        else:
            movie_length = '未知'

        #电影评分
        movie_score = driver.find_elements_by_xpath('//div[@id="interest_sectl"]//strong[@property="v:average"]')
        if(self.isExist(movie_score) and movie_score[0].text !=''):
            movie_score = float(movie_score[0].text)
        else:
            movie_score = 0
        ##评分人数
        movie_score_count = driver.find_elements_by_xpath("//span[@property='v:votes']")
        if(self.isExist(movie_score_count)):
            movie_score_count = int(movie_score_count[0].text)
        else:
            movie_score_count = 0
        ##各星级评分占比
        movie_score_star_per = driver.find_elements_by_xpath("//div[@class='ratings-on-weight']/div/span[@class='rating_per']")
        if(self.isExist(movie_score_star_per)):
            movie_score_star5_per = movie_score_star_per[0].text
            movie_score_star4_per = movie_score_star_per[1].text
            movie_score_star3_per = movie_score_star_per[2].text
            movie_score_star2_per = movie_score_star_per[3].text
            movie_score_star1_per = movie_score_star_per[4].text
        else:
            movie_score_star5_per = 0
            movie_score_star4_per = 0
            movie_score_star3_per = 0
            movie_score_star2_per = 0
            movie_score_star1_per = 0
        ##打包
        movie_score = {
            'movie_score':movie_score,
            'movie_score_count':movie_score_count,
            "stars_5":movie_score_star5_per,
            "stars_4":movie_score_star4_per,
            "stars_3":movie_score_star3_per,
            "stars_2":movie_score_star2_per,
            "stars_1":movie_score_star1_per
        }

        #电影简介
        movie_summary = driver.find_elements_by_xpath("//span[@property='v:summary']")
        if(self.isExist(movie_summary) and movie_summary[0].text !=''):
            movie_summary = movie_summary[0].text
        else:
            movie_summary = '无'

        #推荐电影
        movie_recommendation = []
        movie_recommendation_ele = driver.find_elements_by_xpath('//div[@class="recommendations-bd"]/dl/dt/a/img')
        if(self.isExist(movie_recommendation_ele)):
            for recommendation in movie_recommendation_ele:
                rec_data = {
                    'movie_name': recommendation.get_attribute('alt'),
                    'movie_img': recommendation.get_attribute('src')
                }
                movie_recommendation.append(rec_data)
        #电影评论
        movie_comment_person_img = []
        movie_comment_person_img_eles = driver.find_elements_by_xpath("//a[@class='avator']/img")
        if(self.isExist(movie_comment_person_img_eles) and len(movie_comment_person_img_eles) >= 5):
            for item in movie_comment_person_img_eles[:5]:
                movie_comment_person_img.append(item.get_attribute('src'))
        else:
            movie_comment_person_img = ['https://img1.doubanio.com/icon/user_normal.jpg','https://img1.doubanio.com/icon/user_normal.jpg','https://img1.doubanio.com/icon/user_normal.jpg','https://img1.doubanio.com/icon/user_normal.jpg','https://img1.doubanio.com/icon/user_normal.jpg']
        movie_comment = []
        movie_comment_ele = driver.find_elements_by_xpath('//div[@id="comments-section"]//span[@class="short"]')
        if(self.isExist(movie_comment_ele)):
            for comment in movie_comment_ele:
                movie_comment.append(comment.text)

        index = 0
        movie_comment_info = []
        movie_comment_info_ele = driver.find_elements_by_xpath("//div[@class='comment']/h3")
        if(self.isExist(movie_comment_info_ele)):
            for item in movie_comment_info_ele:
                try:
                    oneScore = item.find_element_by_css_selector("span.comment-info > span.rating").get_attribute('class').split('star')[1].split(' ')[0]
                except:
                    oneScore = '20'
                data = {
                    'movie_comment_person_img':movie_comment_person_img[index],
                    'movie_comment_thumb':item.find_element_by_css_selector("span > span").text,
                    'movie_comment_person_name':item.find_element_by_css_selector("span.comment-info > a").text,
                    'movie_comment_person_score':oneScore,
                    'movie_comment':movie_comment[index]
                }
                movie_comment_info.append(data)
                index += 1


        #电影基础信息表
        data = {
            'movie_name':movie_name,
            'movie_img':movie_img,
        }
        self.mydb.movieBaseInfo.insert(data)
        #表1
        data = {
            'date':own_data["date"],
            'movie_name':movie_name,
            'movie_img':movie_img,
            'boxOffice':own_data["boxOffice"],
            'boxOffice_percentage':own_data['boxOffice_percentage'],
            'arrange':own_data['arrange'],
            'arrange_percentage':own_data['arrange_percentage'],  
            'attendance':own_data["attendance"],
        }
        self.mydb.movieInfo.insert(data)
        if(signal == 1):
            self.mydb.alter.insert(data)
        #情感
        sentiment = []
        for comment in movie_comment:
            sentiment.append(self.get_sentiment(comment))
        #表2
        data = {
            'movie_name':movie_name,
            'movie_date':movie_date,
            'movie_director':movie_director,
            'movie_class':movie_class,
            'movie_length':movie_length,
            'movie_country':movie_country,
            'movie_score':movie_score,
            'movie_summary':movie_summary,
            'movie_recommendation':movie_recommendation,
            'movie_star_info':movie_star_info,
            'movie_comment_info':movie_comment_info,
            'sentiment':sentiment
        }
        self.mydb.movieComment.insert(data)

        print('电影' + movie_name + '爬取完成')
       
    pass